#!/usr/bin/env python
# coding=utf-8

import os, sys, re, traceback, logging
import urllib
import time
import datetime
import getopt
import urllib2
from Route import *

vacation_routeurl = re.compile(r'''<LI><A\s*href="(/Tours/[\S\s]+?)"\s*target=_blank>[\S\s]+?>([\S\s]+?)<[\S\s]+?class=price>([\S\s]+?)<''', re.X|re.S)

route_sights_re = re.compile(r'''<div\s*id=position>[\S\s]+?/Tours/List[\S\s]+?>([\S\s]+?)<''', re.X|re.S)
route_day_re = re.compile(r'''<TD>[\S\s]+?>([\S\s]+?)</TD>''', re.X|re.S)
route_dates_re = re.compile(r'''<TD>[\S\s]+?>([\S\s]+?)</TD>''', re.X|re.S)
route_detail_re = re.compile(r'''<div\s*class=colorbg>([\S\s]+?)</div>''', re.X|re.S)

route_trip_all_re = re.compile(r'''<DIV\s*id=line_content_bar>[\S\s]+?(<DIV[\S\s]+?</DIV>)''', re.X|re.S)
route_trip_c_re = re.compile(r'''<P[\S\s]+?>([\S\s]+?)<''', re.X|re.S)
re_h=re.compile('<[\S\s]*>') #取出信息中夹杂的html标签

http_url = "http://www.ynklzl.com"  #昆明中国国际旅行社
route_telphone = "4007084990"

def do_crawl():
    try:
        server_url_data = [
                ('昆明','跟团游', 'http://www.ynklzl.com/tours/', ),
        ]
        #
        for depcity,routetype,server_url in server_url_data:
            routenum=''
            html = urllib2.urlopen(server_url).read()
            html = getUTF(html)
            vacation_route_url = vacation_routeurl.findall(html)
            for i, route_title, route_price in vacation_route_url:
                try:
                    print "route_url:"+http_url+i
                    route_url = http_url+i
                    #route_url = "http://www.ynklzl.com/Tours/2645.html"
                    #此处开始获取线路信息
                    html = urllib2.urlopen(route_url).read()
                    html = getUTF(html)
                    #
                    route_days = route_day_re.findall(html)
                    if route_days:
                        route_days = route_days[0].split("：")[1].replace("天","")
                    else:
                        route_days = 1
                    print route_days
                    #
                    route_dates=""
                    route_date = route_dates_re.findall(html)
                    if route_date:
                        route_dates = route_date[1].split("：")[1]
                    #print route_dates
                    #
                    route_sights = route_sights_re.findall(html)
                    if route_sights:
                        route_sights = route_sights[0].replace("-",",")
                        #print route_sights
                    #
                    route_detail = route_detail_re.search(html)
                    if route_detail:
                        u=util.strutil(route_detail.group(1))
                        route_detail=u.filter_tags().strip()
                        #print route_detail
                    
                    if route_price:
                        route_price = route_price.split(".")[0]
                        route_price = re.compile(r'''([0-9]+)''', re.X|re.S).findall(route_price)
                        if route_price:
                            route_price = route_price[0]
                    #print "route_title:"+route_title
                    #print "route_price:"+route_price
                    #
                    route_plane = getPlane(html)
                    #print route_plane
                    
                    #
                    route_hotel = ""
                    route_meal = ""
                    route_traffic01 = ""
                    route_tickets = ""
                    route_payown = ""
                    route_trip = route_trip_all_re.findall(html)
                    if route_trip:
                        for route_trips in route_trip_c_re.findall(route_trip[0]):
                            if route_trips.find("住宿")>-1:
                                if route_trips.find("：")>-1:
                                    route_trips = (route_trips.split("："))[1]
                                route_hotel = re_h.sub('',route_trips)
                            if route_trips.find("餐标")>-1:
                                if route_trips.find("：")>-1:
                                    route_trips = (route_trips.split("："))[1]
                                route_meal = re_h.sub('',route_trips)
                            if route_trips.find("交通")>-1 or route_trips.find("用车")>-1:
                                if route_trips.find("：")>-1:
                                    route_trips = (route_trips.split("："))[1]
                                route_traffic01 = re_h.sub('',route_trips)
                            if route_trips.find("门票")>-1:
                                if route_trips.find("：")>-1:
                                    route_trips = (route_trips.split("："))[1]
                                route_tickets = re_h.sub('',route_trips)
                        route_payown = strip_tags(route_trip[1]).strip()
                        #print route_payown+"====="
                    #
                    route_go_back = getTrafficType(route_title)
                    #print route_go_back
                    #入库
                    route = Route()
                    route.site = '46'
                    route.supplier_url = "http://www.ynklzl.com/"
                    route.supplier = "昆明中国国际旅行社"
                    route.title = route_title
                    route.rout_url = route_url
                    route.outcity = depcity
                    route.sights = route_sights
                    route.telphone = route_telphone
                    route.detail = route_detail
                    route.dest = route_sights
                    route.guide = ''
                    #route.type = routetype
                    route.dates = route_dates
                    route.price = int(route_price)
                    route.go_t = route.back_t = route_go_back
                    route.plane = route_plane
                    route.days = route_days

                    route.hotel = route_hotel
                    route.meal = route_meal
                    route.traffic01 = route_traffic01
                    route.tickets = route_tickets
                    route.payown = route_payown
                    
                    getschedules = getSchedules(html)
                    if getschedules=='no':
                        print "==no schedule=="
                    else:
                        route.schedules = getschedules
                        route.updRoute_Schedules()
                    route.updRoute()
                    
                    #print "++++++"
                    #print "++++++"
                    #print "++++++"
                except Exception,e:
                    print traceback.format_exc(sys.exc_info())
                #print routenum+"dddd"
                
    except:
        print traceback.format_exc(sys.exc_info())


Schedules_re = re.compile(r'''<TD\s*align=middle\s*height=25>([\S\s]+?</TD>)</TR>''', re.X|re.S)
Schedules_place_re = re.compile(r'''(<TD[\S\s]+?</TD>)''', re.X|re.S)
Schedules_brief_re = re.compile(r'''<div\s*class=.t_detail_trip_text[^>]+>\s*([\S\s]+?)\s*</div>\s*<div''', re.X|re.S)
Schedules_eat_tra_hot_re = re.compile(r'''<div\s*class=.t_detail_trip_text2[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

def getSchedules(html):
     schedules_place_list = Schedules_re.findall(html)
     if schedules_place_list:
         schedules = range(0,len(schedules_place_list))
         n = 0
         for plane_info in schedules_place_list:
            schedule = Schedule()
            #print plane_info
            if plane_info:
                plane_info = Schedules_place_re.findall(plane_info)
                if plane_info:
                    #print strip_tags(plane_info[0]).strip()
                    #print strip_tags(plane_info[1]).strip()
                    #print strip_tags(plane_info[2]).strip()
                    #print strip_tags(plane_info[3]).strip()
                    schedule.s_brief = strip_tags(plane_info[1]).strip()
                    schedule.s_traffic = strip_tags(plane_info[0]).strip()
                    if strip_tags(plane_info[2]).strip():
                        bldstr = strip_tags(plane_info[2]).strip()
                        schedule.s_brf="1" if bldstr.find('早')>-1 else "0"
                        schedule.s_lunch="1" if bldstr.find('中')>-1 else "0"
                        schedule.s_dinner="1" if bldstr.find('晚')>-1 else "0"
                    if len(plane_info)>2:
                        schedule.s_place= strip_tags(plane_info[3]).strip()
                        #print plane_info
                        #print Schedules_brief_re.findall(html)[n]
                        schedule.s_hotel = strip_tags(plane_info[3]).strip()
                n+=1
                schedule.s_num = n
                        #print "serialnumber:"+str(n)
            else:
                return "no"
            schedules[n-1]=schedule
         return schedules

route_planes_re = re.compile(r'''<TD\s*align=left>([\S\s]+?)</TD>''', re.X|re.S)
def getPlane(html):
     plane_str = ""
     #判断是否含有线路行程天数信息
     plane_info_list = route_planes_re.findall(html)
     if plane_info_list:    
         n = 0
         for plane_info in plane_info_list:
            n+=1
            plane_str = plane_str + "#第" + str(n) + "天# "
            plane_str = plane_str + strip_tags(plane_info).strip()
     return plane_str

def getTrafficType(t):
    t1='-'
    if t.find('汽')>-1:
        t1='汽车'
    if t.find('动')>-1:
        t1='动车'
    if t.find('卧')>-1:
        t1='卧铺'
    if t.find('火车')>-1:
        t1='火车'
    if t.find('船')>-1:
        t1='邮轮'
    if t.find('邮')>-1:
        t1='邮轮'
    if t.find('飞')>-1:
        t1='飞机'
    if t.find('机')>-1:
        t1='飞机'
    return t1

#去掉文本标签
def strip_tags(html):
    from HTMLParser import HTMLParser
    html=html.strip()
    html=html.strip("\n")
    result=[]
    parse=HTMLParser()
    parse.handle_data=result.append
    parse.feed(html)
    parse.close()
    return "".join(result)

def getUTF(gb):
    #uni=unicode(gb,'gb2312')
    utf = gb.decode('gb2312','ignore').encode('utf-8')
    return utf


if __name__ == '__main__':
    script_path = os.path.dirname(os.path.realpath(__file__))
    os.chdir(script_path)
    do_debug = False

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], 'hdno:p:', ['help', 'debug', 'no-cache', 'only-cache', 'provider'])
    except getopt.GetoptError:
        usage()
        sys.exit()

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-d', '--debug'):
            do_debug = True
        elif opt in ('-n', '--no-cache'):
            use_cache = False
        elif opt in ('-o', '--only-cache'):
            if arg.lower() in ('no', 'n', '0'):
                only_cache = False
            else:
                only_cache = True
        elif opt in ('-p', '--provider'):
            pass

    if do_debug:
        import pdb
        pdb.set_trace()
    do_crawl()